import requests
import pandas as pd
import time
import os, sys
import shutil
import json
import numpy as np
from utils.UaUtils import get_user_agent
import shutil
import traceback
from utils.logUtils import Log
from urllib import parse

from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


# root_path = os.path.dirname(os.path.abspath(__file__))
# sys.path.append(root_path)


class RuidGet(object):
    '''
    配置一个唯一的ruid
    props： None
    return：
    '''
    @classmethod
    def get_str_ruid(cls):
        '''
        获取16进制字符串唯一id
        :return:
        '''
        base_time = round(time.mktime(time.strptime('1970-01-02 00:00:00', '%Y-%m-%d %H:%M:%S'))*10**3)

        ruid = round(time.time()*10**3) - base_time
        time.sleep(0.001)
        return str(hex(ruid)).replace('0x', '')

    @classmethod
    def get_int_ruid(cls):
        '''
        获取10进制整数唯一id
        :return:
        '''
        base_time = round(time.mktime(time.strptime('1970-01-02 00:00:00', '%Y-%m-%d %H:%M:%S')) * 10 ** 3)

        ruid = round(time.time() * 10 ** 3) - base_time
        time.sleep(0.001)
        return str(ruid)

def is_nan(value):
    '''
    判断是否为nan
    :return:
    '''
    if not isinstance(value, str) and np.isnan(value):
        return True
    else:
        return False


def find_ext(string):
    if string.endswith(('png', 'PNG')):
        ext = 'png'
    elif string.endswith(('jpg', 'JPG', 'jpeg', 'JPEG')):
        ext = 'jpg'
    elif string.endswith(('gif', 'GIF')):
        ext = 'gif'
    elif string.endswith(('MP3', 'mp3')):
        ext = 'mp3'
    elif string.endswith(('MP4', 'mp4')):
        ext = 'mp4'
    elif string.endswith(('rar', 'RAR')):
        ext = 'rar'
    elif string.endswith(('zip',)):
        ext = 'zip'
    elif string.endswith(('pptx',)):
        ext = 'pptx'
    elif string.endswith(('txt', 'TXT')):
        ext = 'txt'
    elif string.endswith(('doc',)):
        ext = 'doc'
    elif string.endswith(('ppt',)):
        ext = 'ppt'
    elif string.endswith(('pdf', 'PDF')):
        ext = 'pdf'
    elif string.endswith(('epub3.images', 'epub', 'EPUB')):
        ext = 'epub'
    elif string.endswith(('docx',)):
        ext = 'docx'
    elif string.endswith(('xls',)):
        ext = 'xls'
    elif string.endswith(('cnrar',)):
        ext = 'cnrar'
    elif string.endswith(('7z',)):
        ext = '7z'
    elif string.endswith(('ico',)):
        ext = 'ico'
    elif string.endswith(('iso', 'ISO')):
        ext = 'iso'
    elif string.endswith(('xlsx',)):
        ext = 'xlsx'
    elif string.endswith(('gz',)):
        ext = 'gz'
    elif string.endswith(('html',)):
        ext = 'html'
    elif string.endswith(('mobi',)):
        ext = 'mobi'
    else:
        ext = None

    return ext


def get_ext_from_header(headers):
    '''
    从header中提取扩展名
    :return:
    '''
    ext_string = headers.get('Content-Disposition', None)
    ext_string = ext_string[:-1] if (ext_string and ext_string.endswith(('"', "'"))) else ext_string
    if ext_string:
        ext = find_ext(ext_string)
        if ext: return ext

    ext_string = headers.get('Content-Disposition', None)
    ext_string = ext_string[:-2] if (ext_string and ext_string.endswith(('";', "';"))) else ext_string
    if ext_string:
        ext = find_ext(ext_string)
        if ext: return ext

    ext_string = headers.get('Content-Type', None)
    ext_string = ext_string[:-1] if (ext_string and ext_string.endswith(('"', "'"))) else ext_string
    if ext_string:
        ext = find_ext(ext_string)
        if ext: return ext

    return None


if __name__ == '__main__':

    log = Log(mode=Log.ALL)

    try:

        # 使用说明------------------------------------------------
        print('===============================================================')
        print('PS：')
        print('1.文件夹下只放一个excel')
        print('2.excel中需要包含一列“附件url”和一列“附件名”，下载时会读取“附件url”的链接把下载的附件命名为“附件名”')
        print('3.下载前会对excel进行一次备份，保存在bak文件夹下')
        print('4.下载的附件保存在“附件”文件夹下')
        print('5.下载附件时会在excel中新建一列“附件保存名”，将下载好的附件的名称保存在其中（“附件名”和“附件保存名”原则上是一样的，但是出现命名冲突时以附件保存名为准）')
        print('===============================================================')
        time.sleep(1)
        # --------------------------------------------------------

        # # 输入操作路径 ----------------------------------------------------------------
        operate_dir = input('请输入采集好的excel目录（旺柴）:') # D:\PycharmProjects\spiders\图片下载工具\excel
        operate_dir = os.path.abspath(operate_dir)
        # operate_dir = os.path.abspath(r'C:\Users\cxstar46\Desktop\采集矫正补充审核好家伙')
        # -----------------------------------------------------------------------------

        # 采集间隔 -----------------------------
        sleep_time = input('请指定采集间隔时间（单位：秒）:')
        sleep_time = int(sleep_time) if sleep_time else None
        # ------------------------------------
        
        # 是否使用代理 -----------------------------------------------------------------------------------------------------
        proxy_code = input('是否使用代理（如需使用代理，请确保已经打开了小猫咪clash）: \n 1.是 \n 2.否（也可以直接回车，默认不使用代理） \n')
        if proxy_code and proxy_code == '1':
            print('配置代理中...')
            os.environ.update(HTTP_PROXY='http://127.0.0.1:7890', HTTPS_PROXY='http://127.0.0.1:7890')
            print('配置完成！')
        else:
            print('不使用代理噢')
        # ----------------------------------------------------------------------------------------------------------------


        # 读取配置文件 --------------------------------------
        fujian_name_col = '附件名'
        img_url_col = '附件url'
        img_local_path_col = '附件保存名'
        # excel_dir = os.path.abspath('./excel/')
        # img_dir = os.path.abspath('./img/')
        # -------------------------------------------------

        # 创建用来保存封面的img文件夹 --------------------------
        log.info('创建附件文件夹...')
        img_dir_name = '附件'
        img_dir = os.path.join(operate_dir, img_dir_name)
        file_list = os.listdir(operate_dir)
        if img_dir_name not in file_list:
            os.makedirs(img_dir)
        # --------------------------------------------------

        # 检测excel数量，只能放一个，当只有一个excel时，提取它的路径excel_path -------
        log.info('检查路径下的文件格式...')
        excel_name = None
        excel_count = 0
        file_list = os.listdir(operate_dir)
        for file in file_list:
            if file.endswith('.xlsx') or file.endswith('.xlx'):
                excel_count += 1
                excel_name = file
        if excel_count == 0:
            log.info('文件夹下没有excel，按任意键退出...')
            raise Exception(0)
        if excel_count > 1:
            log.info('只能放一个excel，按任意键退出...')
            raise Exception(0)

        # print(excel_name)
        # raise Exception(1212)
        # ----------------------------------------------------------------------

        # 读取excel ----------------------------------------
        excel_path = os.path.join(operate_dir, excel_name)
        # print(excel_path)
        try:
            df = pd.read_excel(excel_path, dtype=str)
        except Exception as e:
            log.error(traceback.format_exc())
            log.info('文件不存在或已损坏...')
            log.info('按任意键退出...')
            raise Exception('123')
        # -------------------------------------------------

        # 备份原始excel表 --------------------------------------------------------------
        log.info('备份excel...')
        bak_dir = 'bak'
        file_list = os.listdir(operate_dir)
        if bak_dir not in file_list:
            os.makedirs(os.path.join(operate_dir, bak_dir))
        bak_excel_path = os.path.join(os.path.join(operate_dir, bak_dir), excel_name)
        shutil.copyfile(excel_path, bak_excel_path)
        # -----------------------------------------------------------------------------

        # # 清空 img文件夹 和 excel表中的 封面保存路径 列 --
        # for file_name in os.listdir(img_dir):
        #     os.remove(img_dir+file_name)
        # df = pd.read_excel(excel_path)
        # df.loc[:, img_local_path_col] = np.nan
        # # print(df.loc[:, '封面本地保存路径'])
        # df.to_excel(excel_path, index=0)
        # # ------------------------------------------

        # 加入 封面保存路径 列 -------------
        if not img_local_path_col in df.columns:
            df.loc[:, img_local_path_col] = np.nan
        # -------------------------------

        # df = df.where(df.notnull(), None)

        for idx in range(df.shape[0]):
            log.info("当前进度： ---------- {} / {} --------------------".format(idx+1, df.shape[0]))

            # if idx < 68579:
            #     continue

            try:
                img_url = df.loc[idx, img_url_col]
            except KeyError as e:
                log.error(traceback.format_exc())
                log.info('没有找到名称为“附件url”的列')
                input('按任意键继续...')
                raise Exception('正常退出')

            # print(img_url)
            if is_nan(img_url):
                log.info('没有附件url： {}'.format(idx))
                continue

            # 断点续采 ---------------------------------
            img_name = df.loc[idx, img_local_path_col]
            # print(img_name, type(img_name))
            if not is_nan(img_name):
                log.info('已采过附件： {}'.format(idx))
                continue
            # ----------------------------------------


            # 阶段性保存 -----------------------------------------------
            if idx % 5 == 0 and idx != 0:
                log.info('----- 阶段性保存,此时关闭程序会导致excel损坏 -----')
                df.to_excel(excel_path, index=0)
            # --------------------------------------------------------


            # 如果指定了 sleep_time ，就歇 sleep_time 秒，防止被封 ----
            if sleep_time:
                time.sleep(sleep_time)
                print(f'休息： {sleep_time} 秒')
            # ---------------------------------------------------


            try:
                headers = {'User-Agent': get_user_agent()}
                log.info('下载文件中： {}'.format(img_url))
                res = requests.get(url=img_url, headers=headers, verify=False, timeout=300)
                print(res.headers)
                # log.info('下载成功')
            except Exception as e:
                log.error(traceback.format_exc())
                df.loc[idx, img_local_path_col] = str(e)
                continue

            if not res.status_code == 200:
                log.info('采集失败，ip可能被封了 或者 目标网站服务器故障： {}'.format(idx))
                continue

            # log.info('采集成功')

            # 规范化文件命名规则 -----------------------------------------------
            new_fujian_name = str(df.loc[idx, fujian_name_col]) \
                .replace('\\xa0', '_') \
                .replace(' ', '_') \
                .replace('\n', '_')\
                .replace('/', '_')\
                .replace('\\', '_')\
                .replace(':', '_')\
                .replace('*', '_')\
                .replace('"', '_')\
                .replace('<', '_')\
                .replace('>', '_')\
                .replace('?', '_')\
                .replace('|', '_')
            # 文件名超过200，后续自动替换为省略号...
            new_fujian_name = new_fujian_name if len(new_fujian_name) < 200 else new_fujian_name[:200]+'...'
            # ---------------------------------------------------------------

            # 从连接中取下载文件的后缀
            ext = find_ext(img_url)
            if not ext:
                # 从返回头中取下载文件的后缀
                ext = get_ext_from_header(res.headers)
                if not ext:
                    log.error('未知文件后缀，跳过')
                    continue

            img_name = f'{new_fujian_name}.{ext}'
            img_path = os.path.join(img_dir, img_name)

            log.info('保存文件中...  {}'.format(img_path))
            with open(img_path, mode='wb') as f:
                f.write(res.content)
            log.info('保存成功')

            df.loc[idx, img_local_path_col] = img_name

        log.info('正在保存...')
        df.to_excel(excel_path, index=0)
        log.info('保存完成')
        input('按任意键继续...')

    except Exception as e:
        log.error(traceback.format_exc())
        # print(traceback.format_exc())
        input('按任意键继续...')
